{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.0.0-alpha0\n",
      "sys.version_info(major=3, minor=7, micro=3, releaselevel='final', serial=0)\n",
      "matplotlib 3.0.3\n",
      "numpy 1.16.2\n",
      "pandas 0.24.2\n",
      "sklearn 0.20.3\n",
      "tensorflow 2.0.0-alpha0\n",
      "tensorflow.python.keras.api._v2.keras 2.2.4-tf\n"
     ]
    }
   ],
   "source": [
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import sklearn\n",
    "import pandas as pd\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "import tensorflow as tf\n",
    "\n",
    "from tensorflow import keras\n",
    "\n",
    "print(tf.__version__)\n",
    "print(sys.version_info)\n",
    "for module in mpl, np, pd, sklearn, tf, keras:\n",
    "    print(module.__name__, module.__version__)\n",
    "    \n",
    "'''\n",
    "具体实现：\n",
    "1. 自己写函数 自定义求导，求偏导，使用计算极小差距值的斜率\n",
    "2. 调用tf.GradentTape()函数，创建tape对象，实现高效求导\n",
    "    tape对象的持久化，使用完毕后 使用del函数 手动删除删除tape对象\n",
    "    求二阶导\n",
    "3. 手动 使用 x.assign_sub()在计算完梯度后，实现变量的梯度下降操作\n",
    "4. 使用内置优化器对象 optimizer，实现变量梯度下降\n",
    "\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7.999999999999119\n"
     ]
    }
   ],
   "source": [
    "def f(x):\n",
    "    return 3. * x ** 2 + 2. * x - 1\n",
    "\n",
    "## 自定以求导过程：计算两个相差极小x值 在函数上的两点之间的斜率\n",
    "def approximate_derivative(f, x, eps=1e-3):\n",
    "    return (f(x + eps) - f(x - eps)) / (2. * eps)\n",
    "\n",
    "print(approximate_derivative(f, 1.))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(8.999999999993236, 41.999999999994486)\n"
     ]
    }
   ],
   "source": [
    "def g(x1, x2):\n",
    "    return (x1 + 5) * (x2 ** 2)\n",
    "\n",
    "## 求偏导，其实就是个分解的过程，把其中一个变量看作是常数\n",
    "def approximate_gradient(g, x1, x2, eps=1e-3):\n",
    "    ## lambda表达式 产生一个函数对象\n",
    "    dg_x1 = approximate_derivative(lambda x: g(x, x2), x1, eps)\n",
    "    dg_x2 = approximate_derivative(lambda x: g(x1, x), x2, eps)\n",
    "    return dg_x1, dg_x2\n",
    "\n",
    "print(approximate_gradient(g, 2., 3.))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(9.0, shape=(), dtype=float32)\n",
      "GradientTape.gradient can only be called once on non-persistent tapes.\n"
     ]
    }
   ],
   "source": [
    "## 使用tf.GradientTape，对变量进行求偏导\n",
    "x1 = tf.Variable(2.0)\n",
    "x2 = tf.Variable(3.0)\n",
    "with tf.GradientTape() as tape:\n",
    "    ## 添加函数对象，只有添加的函数才能被求导\n",
    "    z = g(x1, x2)\n",
    "\n",
    "## 使用tape对其内置的函数 z  对变量x1进行求导\n",
    "dz_x1 = tape.gradient(z, x1)\n",
    "print(dz_x1)\n",
    "\n",
    "## 注意：求导的对象 tape 在这种情况下只能使用一次\n",
    "try:\n",
    "    dz_x2 = tape.gradient(z, x2)\n",
    "except RuntimeError as ex:\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(9.0, shape=(), dtype=float32) tf.Tensor(42.0, shape=(), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "x1 = tf.Variable(2.0)\n",
    "x2 = tf.Variable(3.0)\n",
    "## 添加persistent=True，tape对象持久化\n",
    "with tf.GradientTape(persistent = True) as tape:\n",
    "    z = g(x1, x2)\n",
    "\n",
    "dz_x1 = tape.gradient(z, x1)\n",
    "dz_x2 = tape.gradient(z, x2)\n",
    "print(dz_x1, dz_x2)\n",
    "\n",
    "del tape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<tf.Tensor: id=149, shape=(), dtype=float32, numpy=9.0>, <tf.Tensor: id=161, shape=(), dtype=float32, numpy=42.0>]\n"
     ]
    }
   ],
   "source": [
    "x1 = tf.Variable(2.0)\n",
    "x2 = tf.Variable(3.0)\n",
    "with tf.GradientTape() as tape:\n",
    "    z = g(x1, x2)\n",
    "\n",
    "## 一次性对两个变量求导，返回一个list，\n",
    "## 求偏导的概念，对于有多个变量的函数表达式，固定另一个变量，让函数图形固定在一个平面上，进行降维。转化为一般的求法\n",
    "dz_x1x2 = tape.gradient(z, [x1, x2])\n",
    "\n",
    "print(dz_x1x2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[None, None]\n"
     ]
    }
   ],
   "source": [
    "## 但是使用tf.GradientTape() ，针对的对象是tf.Variable，是变量\n",
    "## 之间将constant传入 将无法求导\n",
    "x1 = tf.constant(2.0)\n",
    "x2 = tf.constant(3.0)\n",
    "with tf.GradientTape() as tape:\n",
    "    z = g(x1, x2)\n",
    "\n",
    "dz_x1x2 = tape.gradient(z, [x1, x2])\n",
    "\n",
    "print(dz_x1x2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<tf.Tensor: id=192, shape=(), dtype=float32, numpy=9.0>, <tf.Tensor: id=204, shape=(), dtype=float32, numpy=42.0>]\n"
     ]
    }
   ],
   "source": [
    "x1 = tf.constant(2.0)\n",
    "x2 = tf.constant(3.0)\n",
    "with tf.GradientTape() as tape:\n",
    "    ## 解决constant求导，使用tape.watch()将该变量也纳入观察中，即可进行求导操作\n",
    "    tape.watch(x1)\n",
    "    tape.watch(x2)\n",
    "    z = g(x1, x2)\n",
    "\n",
    "dz_x1x2 = tape.gradient(z, [x1, x2])\n",
    "\n",
    "print(dz_x1x2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=261, shape=(), dtype=float32, numpy=13.0>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## 两个函数同时对一个变量进行求导\n",
    "x = tf.Variable(5.0)\n",
    "with tf.GradientTape() as tape:\n",
    "    ## 从这里看出，tape需要的只是一个函数表达式\n",
    "    ## 可以是直接从外界传入，也可以是直接定义表达式；或者是lambda表达式\n",
    "    z1 = 3 * x\n",
    "    z2 = x ** 2\n",
    "tape.gradient([z1, z2], x)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[None, <tf.Tensor: id=324, shape=(), dtype=float32, numpy=6.0>], [<tf.Tensor: id=378, shape=(), dtype=float32, numpy=6.0>, <tf.Tensor: id=361, shape=(), dtype=float32, numpy=14.0>]]\n"
     ]
    }
   ],
   "source": [
    "## 求二阶导数，先对x1求导 再对 x2求导\n",
    "x1 = tf.Variable(2.0)\n",
    "x2 = tf.Variable(3.0)\n",
    "with tf.GradientTape(persistent=True) as outer_tape:\n",
    "    with tf.GradientTape(persistent=True) as inner_tape:\n",
    "        z = g(x1, x2)\n",
    "    inner_grads = inner_tape.gradient(z, [x1, x2])\n",
    "outer_grads = [outer_tape.gradient(inner_grad, [x1, x2])\n",
    "               for inner_grad in inner_grads]\n",
    "print(outer_grads)\n",
    "del inner_tape\n",
    "del outer_tape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<tf.Variable 'Variable:0' shape=() dtype=float32, numpy=-0.3333333>\n"
     ]
    }
   ],
   "source": [
    "## 在求完导数之后，将变量在导数方向上进行梯度下降操作\n",
    "## 相当于optimizer函数的 功能\n",
    "learning_rate = 0.1\n",
    "x = tf.Variable(0.0)\n",
    "\n",
    "for _ in range(100):\n",
    "    with tf.GradientTape() as tape:\n",
    "        z = f(x)\n",
    "    dz_dx = tape.gradient(z, x)\n",
    "    ## 变量减少操作：assign_sub()将变量减少指定的数量\n",
    "    x.assign_sub(learning_rate * dz_dx)\n",
    "print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<tf.Variable 'Variable:0' shape=() dtype=float32, numpy=-0.3333333>\n"
     ]
    }
   ],
   "source": [
    "learning_rate = 0.1\n",
    "x = tf.Variable(0.0)\n",
    "\n",
    "## 这个优化器对象 optimizer，相当于上 assign_sub的一个封装\n",
    "## 将各个变量在其梯度方向上进行 减少操作\n",
    "optimizer = keras.optimizers.SGD(lr = learning_rate)\n",
    "\n",
    "for _ in range(100):\n",
    "    with tf.GradientTape() as tape:\n",
    "        z = f(x)\n",
    "    dz_dx = tape.gradient(z, x)\n",
    "    ## 传入数据的格式，list中的每个item都是一个tuple，分别填入(导数,变量)\n",
    "    optimizer.apply_gradients([(dz_dx, x)])\n",
    "print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
